#!/usr/bin/env python
# encoding: utf-8
"""
#-------------------------------------------------------------------#
#                   CONFIDENTIAL --- CUSTOM STUDIOS                 #     
#-------------------------------------------------------------------#
#                                                                   #
#                   @Project Name : Globallawonline                #
#                                                                   #
#                   @File Name    : cnkispider.py                      #
#                                                                   #
#                   @Programmer   : 李建                            #
#                                                                   #  
#                   @Start Date   : 2021/8/6 9:04                 #
#                                                                   #
#                   @Last Update  : 2021/8/6 9:04                 #
#                                                                   #
#-------------------------------------------------------------------#
# Classes:知网国际条约数据采集蜘蛛                                                          #
#                                                                   #
#-------------------------------------------------------------------#
"""
import hashlib
import os
import time
import re
import scrapy
import pandas as pd
from scrapy import signals
from scrapy.http import HtmlResponse
from selenium.webdriver.chrome.options import Options

from ..items import MyFileItem
from selenium import webdriver


class Spider(scrapy.Spider):
    name = 'cnkispider'
    allowed_domains = 'https://lawnew.cnki.net/'
    # 知网法律法规库网址
    start_urls = ['https://lawnew.cnki.net/kns/brief/result.aspx?dbprefix=CLKLK']
    id = 0
    Current_page = ''

    def __init__(self, **kwargs):
        super(Spider, self).__init__(**kwargs)
        # chrome_options = Options()
        # chrome_options.add_argument('--no-sandbox')
        # chrome_options.add_argument('--disable-dev-shm-usage')
        # chrome_options.add_argument('--headless')
        # self.driver = webdriver.Chrome(chrome_options=options)  # 调用本地的谷歌浏览器
        chrome_options = Options()
        chrome_options.binary_location = r'C:\Program Files\Google\Chrome\Application\chrome.exe'
        self.driver = webdriver.Chrome(executable_path=os.path.abspath("C:\ProgramData\Anaconda3\chromedriver.exe"), options=chrome_options)

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(Spider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.closeSpider, signals.spider_closed)
        return spider

    def closeSpider(self):
        self.driver.quit()  # 关闭浏览器

    def parse(self, response):
        name = 'XCS13539'
        password = 'gsgk105'
        self.driver.find_element_by_xpath('//*[@id="header"]/div[2]/ul/div[2]/li[1]/a').click()
        self.driver.find_element_by_xpath('//*[@id="TextBoxUserName"]').send_keys(name)
        self.driver.find_element_by_xpath('//*[@id="TextBoxPwd"]').send_keys(password)
        self.driver.find_element_by_css_selector('#Button1').click()
        data = pd.read_excel('E:\工作记录\工作内容\“一带一路”项目\各国专家提供的法律资源资料\国际分社国际条约\东盟十国法规清单20210719.xls', '英文全文')
        for i in range(0,len(data)):
            cnkiid = str(data.iloc[i,0])
            cnkititle = str(data.iloc[i,1])
            isexit = str(data.iloc[i,8])
            if isexit == 'nan':
                self.driver.find_element_by_xpath('//*[@id="txt_1_value1"]').clear()
                self.driver.find_element_by_xpath('//*[@id="txt_1_value1"]').send_keys(cnkititle)
                self.driver.find_element_by_css_selector('#btnSearch').click()
                time.sleep(5)
                searchhandle = self.driver.current_window_handle
                resulthtml = self.driver.find_element_by_xpath('//*[@id="iframeResult"]').get_attribute('src')
                js = 'window.open("%s");' % resulthtml
                self.driver.execute_script(js)
                allHandles = self.driver.window_handles
                time.sleep(2)
                for handle in allHandles:
                    if handle != searchhandle:
                        self.driver.switch_to.window(handle)
                        resulthandle = handle
                origin_code = self.driver.page_source
                resresponse = HtmlResponse(url=self.driver.current_url, encoding='utf8', body=origin_code, request=response.url)
                trs = resresponse.xpath('//*[@id="ctl00"]/table/tbody/tr[2]/td/table/tbody/tr')
                if len(trs):
                    for j in range(1,len(trs)+1):
                        trxpath = '//*[@id="ctl00"]/table/tbody/tr[2]/td/table/tbody/tr[%s]/td[5]//text()' % j
                        Timeliness = resresponse.xpath(trxpath)  # 时效性
                        Timelines = ''
                        if len(Timeliness):
                            for Timelinessl in Timeliness:
                                Timelines = Timelines + Timelinessl.get().strip()
                        Timelines = Timelines.strip()
                        if Timelines == "现行有效":     # 现行有效的数据才会采集
                            dexpath = '//*[@id="ctl00"]/table/tbody/tr[2]/td/table/tbody/tr[%s]/td[2]/a' % j
                            self.driver.find_element_by_xpath(dexpath).click()
                            time.sleep(3)
                            allHandles = self.driver.window_handles
                            for handle in allHandles:
                                if handle != searchhandle and handle != resulthandle:
                                    self.driver.switch_to.window(handle)
                                    detailhandle = handle
                            origin_code = self.driver.page_source
                            deresponse = HtmlResponse(url=self.driver.current_url, encoding='utf8', body=origin_code,
                                                       request=response.url)
                            if deresponse.url.find(cnkiid)>0:     # 判断该条数据的知网id是否是目标数据
                                # 发布机关
                                publishcountryS = deresponse.xpath('//*[@id="main"]//div[@class="author"]/p[1]//text()')
                                publishcountry = ''
                                if len(publishcountryS):
                                    for publishcountrySl in publishcountryS:
                                        publishcountry = publishcountry + publishcountrySl.get().strip()
                                publishcountry = publishcountry.strip()
                                # 发表日期
                                publishDateS = deresponse.xpath('//*[@id="main"]//div[@class="author"]/p[2]//text()')
                                publishDate = ''
                                if len(publishDateS):
                                    for publishDateSl in publishDateS:
                                        publishDate = publishDate + publishDateSl.get().strip()
                                publishDate = publishDate.strip()
                                # 生效日期
                                effictiveDateS = deresponse.xpath('//*[@id="main"]//div[@class="author"]/p[3]//text()')
                                effictiveDate = ''
                                if len(effictiveDateS):
                                    for effictiveDateSl in effictiveDateS:
                                        effictiveDate = effictiveDate + effictiveDateSl.get().strip()
                                effictiveDate = effictiveDate.strip()
                                try:
                                    self.driver.find_element_by_css_selector('#nav > ul > li.pdf > a').click()
                                    data.iloc[i, 5] = publishcountry
                                    data.iloc[i, 6] = publishDate
                                    data.iloc[i, 7] = effictiveDate
                                    data.iloc[i, 8] = cnkititle + '.pdf'
                                    data.to_excel(
                                        r'E:\工作记录\工作内容\“一带一路”项目\各国专家提供的法律资源资料\国际分社国际条约\东盟十国法规清单20210719_英文全文.xls')
                                except Exception as e:
                                    print(str(e))
                                time.sleep(8)
                                self.driver.close()
                                self.driver.switch_to.window(resulthandle)
                                break
                            else:
                                self.driver.close()
                                self.driver.switch_to.window(resulthandle)
                self.driver.close()
                self.driver.switch_to.window(searchhandle)